package org.net.util;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.swing.text.EditorKit;
import javax.swing.text.SimpleAttributeSet;
import javax.swing.text.html.HTML;
import javax.swing.text.html.HTMLDocument;
import javax.swing.text.html.HTMLEditorKit;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.net.vo.SiteVo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.web.core.service.ResourceService;
import org.web.entity.Resources;

/**
 * @Author : yingx
 * @CreateTime : 2011-9-28 下午05:00:07
 * @Pageage : org.net.util
 */
public class AnalysisContentUtil {
	
	private static Logger log = LoggerFactory.getLogger(AnalysisContentUtil.class);
	
	public static Map<String,List<SiteVo>> map = new HashMap<String,List<SiteVo>>();

	public static Map<String, List<SiteVo>> getMap() {
		return map;
	}

	public static HashSet<String> set = new HashSet<String>();

	public static HashSet<String> getSet() {
		return set;
	}

	/**
	 * 判断是否有留言标识
	 * @param responseBody
	 * @return
	 */
	public static boolean canLeaveComment(String responseBody){
//		if(!StringUtil.isEmpty(responseBody) && ((StringUtil.ignoreIndexOf(responseBody,"If you have a TypeKey or TypePad account, please Sign In",0,true)!=-1)
//				|| StringUtil.ignoreIndexOf(responseBody,"If you have a TypeKey or TypePad account, please Sign In",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"Comment below or sign in with TypePad Facebook Twitter and more",0,true)!=-1
//				|| StringUtil.ignoreIndexOf(responseBody,"There may be a delay before your comment is published",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"Displayed next to your comments. Post a new comment",0,true)!=-1
//				|| StringUtil.ignoreIndexOf(responseBody,"Comments by Joomla Professional Solutions",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"Powered by !JoomlaComment",0,true)!=-1
//				|| StringUtil.ignoreIndexOf(responseBody,"Please note, comments will be published immediately",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"Enter YourName@YourKeywords in the Name field to take advantage",0,true)!=-1
//				|| StringUtil.ignoreIndexOf(responseBody,"Powered by: BlogEngine.NET",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"To help keep spam off our site, please enter the letter",0,true)!=-1
//				|| StringUtil.ignoreIndexOf(responseBody,"Empty lines separate paragraphs. Use @code@ for inline code, and > code for code blocks",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"You must read and type the 5 chars within 0..9 and A..F, and submit the form",0,true)!=-1
//				|| StringUtil.ignoreIndexOf(responseBody,"Your comment was successfully posted",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"This question is for testing whether you are a human visitor and to prevent automated spam submissions",0,true)!=-1
//				|| StringUtil.ignoreIndexOf(responseBody,"blog comments powered by DISQUS Add New Comment",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"Spam Protection by WP-SpamFree Plugin LEAVE A REPLY",0,true)!=-1
//				|| StringUtil.ignoreIndexOf(responseBody,"Powered by Movable Type Pro",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"Spam Protection by WP-SpamFree",0,true)!=-1
//				|| StringUtil.ignoreIndexOf(responseBody,"Spam Protection by RDV",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"Leave a comment",0,true)!=-1
//				|| StringUtil.ignoreIndexOf(responseBody,"Leave a Reply",0,true)!=-1 || StringUtil.ignoreIndexOf(responseBody,"Post your response",0,true)!=-1
//				|| StringUtil.ignoreIndexOf(responseBody, "Post a comment", 0, true)!=-1) && StringUtil.ignoreIndexOf(responseBody, "Sign in to comment",0,true)==-1){
		if(!StringUtil.isEmpty(responseBody) && (StringUtil.ignoreIndexOf(responseBody, "Post a comment", 0, true)!=-1 || 
				StringUtil.ignoreIndexOf(responseBody, "Write comment", 0, true)!=-1 || StringUtil.ignoreIndexOf(responseBody, "Add new comment", 0, true)!=-1 ||
				StringUtil.ignoreIndexOf(responseBody, "Leave a Comment", 0, true)!=-1 || StringUtil.ignoreIndexOf(responseBody, "Leave a Reply", 0, true)!=-1 ||
				StringUtil.ignoreIndexOf(responseBody, "drop a comment", 0, true)!=-1 || StringUtil.ignoreIndexOf(responseBody, "Add a Comment", 0, true)!=-1 || 
				StringUtil.ignoreIndexOf(responseBody, "Post a new comment", 0, true)!=-1 || StringUtil.ignoreIndexOf(responseBody, "Add Your Comment", 0, true)!=-1 || 
				StringUtil.ignoreIndexOf(responseBody, "Leave a Comment", 0, true)!=-1 || StringUtil.ignoreIndexOf(responseBody, "Leave a Comment", 0, true)!=-1 || 
				StringUtil.ignoreIndexOf(responseBody, "Leave a Comment", 0, true)!=-1 || StringUtil.ignoreIndexOf(responseBody, "Leave a Comment", 0, true)!=-1 ||
				StringUtil.ignoreIndexOf(responseBody, "Add comment", 0, true)!=-1 || StringUtil.ignoreIndexOf(responseBody, "Write comment", 0, true)!=-1 || 
				StringUtil.ignoreIndexOf(responseBody, "Speak Your Mind", 0, true)!=-1 || StringUtil.ignoreIndexOf(responseBody, "Post your comment", 0, true)!=-1 || 
				StringUtil.ignoreIndexOf(responseBody, "Post your response", 0, true)!=-1 || 
				StringUtil.ignoreIndexOf(responseBody, "Add comment", 0, true)!=-1 ) && StringUtil.ignoreIndexOf(responseBody, "Sign in to comment",0,true)==-1 && StringUtil.ignoreIndexOf(responseBody, "Login in to comment",0,true)==-1){
			return true;
		}else{
			return false;
		}
	}
	
	/**
	 * 利用YAHOO查找站的外链接
	 * @param content
	 * @return
	 */
	@SuppressWarnings("unchecked")
	public static List<String> fetchAllHrefFromYahooSite(String content){
		List results = new ArrayList();
		try{
		if(StringUtil.isEmpty(content))    return results;
		EditorKit kit = new HTMLEditorKit();
	    HTMLDocument doc = (HTMLDocument) kit.createDefaultDocument();
	    kit.read(new ByteArrayInputStream(content.toString().getBytes()), doc, 0);

	    HTMLDocument.Iterator it = doc.getIterator(HTML.Tag.A);
	    while (it.isValid()) {
	      SimpleAttributeSet s = (SimpleAttributeSet) it.getAttributes();

	      String link = (String) s.getAttribute(HTML.Attribute.HREF);
	      String title = (String) s.getAttribute(HTML.Attribute.TITLE);
	      if (link != null && !StringUtil.isEmpty(title) && link.equalsIgnoreCase(title)) {
	        results.add(link);
	      }
	      it.next();
	    }}catch(Exception e){
	    	
	    }
	    return results;
	}
	
	
	/**
	 * 查找该页面非本站的链接数量
	 * @param currentUrl
	 * @param content
	 * @return
	 */
	public static int countAllHrefExceptSelf(String currentUrl ,String content){
		int count =0;
		if(StringUtil.isEmpty(content))     return count;
		try{
			EditorKit kit = new HTMLEditorKit();
		    HTMLDocument doc = (HTMLDocument) kit.createDefaultDocument();
		    
		    doc.putProperty("IgnoreCharsetDirective", new Boolean(true));
		    kit.read(new ByteArrayInputStream(content.toString().getBytes()), doc, 0);

		    HTMLDocument.Iterator it = doc.getIterator(HTML.Tag.A);
		    String currentDomain = AnalysisContentUtil.getItsDomainName(currentUrl);
		    List<SiteVo> site = new ArrayList<SiteVo>();
		    while (it.isValid()) {
			      SimpleAttributeSet s = (SimpleAttributeSet) it.getAttributes();
			      //取出页面上所有的链接
			      String link = (String) s.getAttribute(HTML.Attribute.HREF);
			      if (link != null) {
			         if(!StringUtil.isEmpty(currentDomain) && link.indexOf(currentDomain)==-1){
			        	 int start = it.getStartOffset();
				         int end = it.getEndOffset();
				         String name = doc.getText(start, end-start);
				         if(!StringUtil.isEmpty(name) && name.indexOf("http://")==-1 && (name.toLowerCase().indexOf("sale")!=-1 || name.toLowerCase().indexOf("hats")!=-1 || name.toLowerCase().indexOf("nike")!=-1
				        		 || name.toLowerCase().indexOf("bag")!=-1 || name.toLowerCase().indexOf("nfl")!=-1 || name.toLowerCase().indexOf("shoes")!=-1
				        		 || name.toLowerCase().indexOf("jersey")!=-1 || name.toLowerCase().indexOf("cheap")!=-1
				        		 || name.toLowerCase().indexOf("ugg")!=-1 || name.toLowerCase().indexOf("gucci")!=-1 || name.toLowerCase().indexOf("jordan")!=-1 ||
				        		 name.toLowerCase().indexOf("jacket")!=-1 || name.toLowerCase().indexOf("discount")!=-1 || name.toLowerCase().indexOf("discount")!=-1 ||
				        		 name.toLowerCase().indexOf("replica")!=-1 || name.toLowerCase().indexOf("goose")!=-1 ||name.toLowerCase().indexOf("lv")!=-1 ||
				        		 name.toLowerCase().indexOf("watches")!=-1 || name.toLowerCase().indexOf("dresses")!=-1 || name.toLowerCase().indexOf("puma")!=-1 ||
				        		 name.toLowerCase().indexOf("pens")!=-1 || name.toLowerCase().indexOf("jeans")!=-1 || name.toLowerCase().indexOf("store")!=-1 ||
				        		 name.toLowerCase().indexOf("polo")!=-1 || name.toLowerCase().indexOf("dvd")!=-1 || name.toLowerCase().indexOf("shop")!=-1)){
					         SiteVo sv = new SiteVo();
				        	 sv.setInnerText(name);
				        	 sv.setOpendUrl(currentUrl);
				        	 sv.setUrl("http://www."+getItsDomainName(link));
				        	 if(link.indexOf("http://")!=-1 && !set.contains(sv.getUrl())){
					        	 site.add(sv);
					         }
					         set.add(sv.getUrl());
				         }
				         count++;
			         }
			      }
			      it.next();
			    }
//			    Collections.sort(site,new Comparator<SiteVo>(){
//					 public int compare(SiteVo o1, SiteVo o2){
//					      return o2.getOpenedTime().compareTo(o1.getOpenedTime());
//					 }
//				});
		        Collections.reverse(site);
		        if(site.size()>2) site = site.subList(0, 2);    
			    map.put(currentUrl, site);
			    for(SiteVo v: site){
			    	System.out.println(v.getUrl());
			    }
		}catch(Exception e){
			
		}
		return count;
	}
	
	@SuppressWarnings("static-access")
	public static int countAllHrefExceptSelfInputSteam(String currentUrl,ResourceService resourceService){
		int count =0;
		if(StringUtil.isEmpty(currentUrl))     return count;
		HttpClient httpClient = FetchUtil.initClient();
		FetchUtil.setProxy(httpClient); 
		GetMethod getMethod = new GetMethod(currentUrl);
		try{
	        log.info("count link number in："+currentUrl);
			int statusCode = httpClient.executeMethod(getMethod);
			if (statusCode != HttpStatus.SC_OK) {
				System.err.println("Method failed: " + getMethod.getStatusLine());
				log.error("can't get page content to analysic");
				return 0;
			}
			httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(30000);  
		    getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT,30000); 
		    
			EditorKit kit = new HTMLEditorKit();
		    HTMLDocument doc = (HTMLDocument) kit.createDefaultDocument();
		    
		    doc.putProperty("IgnoreCharsetDirective", new Boolean(true));
//		    kit.read(new InputStreamReader(getMethod.getResponseBodyAsStream()), doc, 0);
//		    ByteArrayInputStream bytes = new ByteArrayInputStream(getMethod.getResponseBody());
		    
		    BufferedReader br = new BufferedReader(new InputStreamReader(getMethod.getResponseBodyAsStream()));
		    Thread.currentThread().sleep(100);   // 数据有足够的时间写到缓冲区
    		String tempbf;
    		StringBuffer html=new StringBuffer();
    		while((tempbf=br.readLine())!=null){
    		html.append(tempbf);
    		} 
    		ByteArrayInputStream bytes = new ByteArrayInputStream(html.toString().getBytes());
		    kit.read(bytes, doc, 0);
		    
		    HTMLDocument.Iterator it = doc.getIterator(HTML.Tag.A);
		    String currentDomain = AnalysisContentUtil.getItsDomainName(currentUrl);
		    List<SiteVo> site = new ArrayList<SiteVo>();
		    while (it.isValid()) {
		      SimpleAttributeSet s = (SimpleAttributeSet) it.getAttributes();
		      //取出页面上所有的链接
		      String link = (String) s.getAttribute(HTML.Attribute.HREF);
		      if (link != null) {
		         if(!StringUtil.isEmpty(currentDomain) && link.indexOf(currentDomain)==-1){
		        	 int start = it.getStartOffset();
			         int end = it.getEndOffset();
			         String name = doc.getText(start, end-start);
			         if(!StringUtil.isEmpty(name) && name.indexOf("http://")==-1 && name.indexOf("/") ==-1 && (name.toLowerCase().indexOf("sale")!=-1 || name.toLowerCase().indexOf("hats")!=-1 || name.toLowerCase().indexOf("nike")!=-1
			        		 || name.toLowerCase().indexOf("bag")!=-1 || name.toLowerCase().indexOf("nfl")!=-1 || name.toLowerCase().indexOf("shoes")!=-1
			        		 || name.toLowerCase().indexOf("jersey")!=-1 || name.toLowerCase().indexOf("cheap")!=-1
			        		 || name.toLowerCase().indexOf("ugg")!=-1 || name.toLowerCase().indexOf("gucci")!=-1 || name.toLowerCase().indexOf("jordan")!=-1 ||
			        		 name.toLowerCase().indexOf("jacket")!=-1 || name.toLowerCase().indexOf("discount")!=-1 || name.toLowerCase().indexOf("discount")!=-1 ||
			        		 name.toLowerCase().indexOf("replica")!=-1 || name.toLowerCase().indexOf("goose")!=-1 ||name.toLowerCase().indexOf("lv")!=-1 ||
			        		 name.toLowerCase().indexOf("watches")!=-1 || name.toLowerCase().indexOf("dresses")!=-1 || name.toLowerCase().indexOf("puma")!=-1 ||
			        		 name.toLowerCase().indexOf("pens")!=-1 || name.toLowerCase().indexOf("jeans")!=-1 || name.toLowerCase().indexOf("store")!=-1 ||
			        		 name.toLowerCase().indexOf("polo")!=-1 || name.toLowerCase().indexOf("dvd")!=-1 || name.toLowerCase().indexOf("shop")!=-1)){
			        	 SiteVo sv = new SiteVo();
			        	 sv.setInnerText(name);
			        	 sv.setOpendUrl(currentUrl);
			        	 sv.setUrl("http://www."+getItsDomainName(link));
			        	 if(link.indexOf("http://")!=-1 && !set.contains(sv.getUrl())){
				        	 site.add(sv);
				         }
				         set.add(sv.getUrl());
			         }
			         count++;
		         }
		      }
		      it.next();
		    }
		    log.info("find link count ："+count+" in "+currentUrl);
		    if(count<1000){
		    Collections.reverse(site);
		    if(site.size()>5) site = site.subList(0, 5);    //取倒数的5个
//		    map.put(currentUrl, site);
		    final List<SiteVo> svs = site;
		    final ResourceService service = resourceService;
		    new Thread(){
		    	public void run(){
		    		for(SiteVo sv :svs) {
						try{
					    log.info("find source："+sv.getUrl()+"---"+sv.getInnerText()+"--"+sv.getOpendUrl()+"\r\n");
					    Resources res = new Resources();
					    res.setCreatedDate(new Date());
					    res.setFromWhere(sv.getOpendUrl());
					    res.setUrl(sv.getUrl());
					    res.setInnerText(sv.getInnerText());
					    res.setIsCrawled("N");
					    service.save(res);
					    this.currentThread().sleep(100);
						}catch(Exception e){
							e.printStackTrace();
							continue;
						}
					}
		    	}
		    }.start();
		    }
		}catch(Exception e){
			e.printStackTrace();
			return 0;
		}finally{
			getMethod.releaseConnection();
		}
		
		return count;
	}
	
	/**
	 * 提取网址的域名
	 * @param url
	 * @return
	 */
	public static String getItsDomainName(String url){
		if(StringUtil.isEmpty(url))    return "";
		String s1 = "";
		if(url.indexOf("?")==-1 && url.replaceAll("http://", "").indexOf("/")==-1){
			s1 = url;
		}else{
			Matcher m=Pattern.compile("(?<=(http://)).*?(?=(\\/|\\?))").matcher(url);
		    if(m.find()){
	          s1=m.group();
		    }
		}
		s1 = s1.replaceAll("http://", "");
	    if(!StringUtil.isEmpty(s1)){
      	  String[] ts = s1.split("\\.");
      	  if(ts.length>2){
      		  if(s1.indexOf("com.cn")==-1){
      		    return ts[1]+"."+ts[2];
      		  }else{
      			return ts[ts.length-3] +".com.cn"; 
      		  }
      	  }else{
      		  return s1;
      	  }
        }
		return "";
	}
	
	public static void main(String[] args){
		System.out.println("http://www.300l600.de/bc?abc&id=2/43,0,0,1,0".split("\\.").length);
		System.out.println(getItsDomainName("http://sec.300l600.cn/bc?abc&id=2/43,0,0,1,0"));
		System.out.println(getItsDomainName("http://300l600.cns"));
		System.out.println(getItsDomainName("http://www.000000.cn"));
		System.out.println(getItsDomainName("http://www.11111.cn/aaaa"));
		System.out.println(getItsDomainName("http://www.2121212.cn?aaa=bbb"));
		String str ="<a href='http://abc.com' title='abc'>abcinner</a>adsadfsdf<a href='http://cde.cn' title='cde'>cdeinner</a>";
		countAllHrefExceptSelf("http://abcff.com/ab","<html><head><title>abcugg</title></head><body><a href='http://www.aaa.com'>aaaugg</a>----" +
				"<a href='http://www.aaa1.com'>aaa1ugg</a><br/><a href='http://www.aaa2.com'>aaa2ugg</a><br/><a href='http://www.aaa3.com'>aaa3ugg</a></body></html>");
	}
}
